#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
快乐8数据采集脚本
"""
import requests
from bs4 import BeautifulSoup
import pandas as pd
import csv
import os
import logging
from datetime import datetime

# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

def fetch_latest_kl8_data(output_file):
    """从官方网站获取最新的快乐8开奖数据
    
    Args:
        output_file: 输出CSV文件路径
    """
    # 使用官方API获取数据
    url = "https://www.cwl.gov.cn/cwl_admin/front/cwlkj/search/kjxx/findDrawNotice?name=kl8&issueCount=100"
    
    try:
        # 发送请求获取数据
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
            'Referer': 'https://www.cwl.gov.cn/',
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'X-Requested-With': 'XMLHttpRequest'
        }
        response = requests.get(url, headers=headers, timeout=30)
        response.raise_for_status()
        
        # 解析JSON数据
        data = response.json()
        logger.info(f"从API获取到数据，状态: {data.get('state')}, 消息: {data.get('message')}")
        
        # 检查返回状态
        if data.get('state') != 0:
            logger.error(f"API返回错误状态: {data.get('state')}, 消息: {data.get('message')}")
            return
        
        # 准备数据列表
        records = []
        
        # 解析每期开奖数据
        for item in data.get('result', []):
            # 提取开奖信息
            issue = item.get('code', '')  # 期号
            date = item.get('date', '')   # 开奖日期
            numbers = item.get('red', '') # 开奖号码
            
            # 处理日期格式，确保是标准格式
            if date:
                # 如果日期包含星期几的中文字符，移除它们
                import re
                pattern = r'(\d{4}-\d{1,2}-\d{1,2})\([一二三四五六日]\)'
                match = re.match(pattern, date)
                if match:
                    date = match.group(1)
                # 处理斜杠格式的日期
                elif '/' not in date and '-' in date:
                    # 保持原格式
                    pass
                elif '/' in date:
                    # 将斜杠格式转换为连字符格式
                    parts = date.split(' ')[0].split('/')
                    if len(parts) == 3:
                        date = f"{parts[0]}/{parts[1]}/{parts[2]}"
            
            # 处理号码数据
            if numbers:
                # 将逗号分隔的号码转换为列表
                try:
                    number_list = [int(n) for n in numbers.split(',')]
                    
                    # 确保有20个号码
                    if len(number_list) == 20:
                        # 按升序排列
                        number_list.sort()
                        records.append([issue, date] + number_list)
                        logger.info(f"解析期号 {issue} 数据成功: {numbers}")
                    else:
                        logger.warning(f"期号 {issue} 的号码数量不正确: {len(number_list)}, 号码: {numbers}")
                except ValueError as e:
                    logger.warning(f"期号 {issue} 的号码格式错误: {numbers}, 错误: {e}")
        
        # 保存到CSV文件
        if records:
            # 读取现有数据（如果文件存在）
            existing_records = []
            if os.path.exists(output_file):
                with open(output_file, 'r', newline='', encoding='utf-8') as csvfile:
                    reader = csv.reader(csvfile)
                    existing_records = list(reader)
                # 移除表头
                if existing_records:
                    existing_records = existing_records[1:]
            
            # 合并现有数据和新数据
            all_records = existing_records + records
            
            # 按期号排序，确保数据是正序的（从旧到新）
            all_records.sort(key=lambda x: int(x[0]))  # 按照期号列（索引为0）进行排序
            
            # 去重，保留最新的数据
            unique_records = []
            seen_issues = set()
            for record in reversed(all_records):  # 从新到旧遍历
                issue = record[0]
                if issue not in seen_issues:
                    seen_issues.add(issue)
                    unique_records.append(record)
            unique_records.reverse()  # 恢复为从旧到新的顺序
            
            # 写入数据
            with open(output_file, 'w', newline='', encoding='utf-8') as csvfile:
                writer = csv.writer(csvfile)
                # 写入表头
                header = ['期号', '开奖日期'] + [f'{i}号' for i in range(1, 21)]
                writer.writerow(header)
                # 写入数据
                writer.writerows(unique_records)
            
            logger.info(f"成功获取 {len(records)} 期开奖数据并保存到 {output_file}（已按期号正序排列）")
            return len(records)  # 返回获取到的记录数
        else:
            logger.warning("未获取到有效的开奖数据")
            return 0
            
    except requests.RequestException as e:
        logger.error(f"网络请求失败: {e}")
        return 0
    except Exception as e:
        logger.error(f"数据获取过程中发生错误: {e}")
        import traceback
        logger.error(f"详细错误信息: {traceback.format_exc()}")
        return 0

def fetch_kl8_data_from_alternative_source(output_file):
    """从备用数据源获取快乐8开奖数据
    
    Args:
        output_file: 输出CSV文件路径
    """
    # 备用数据源URL（示例）
    url = "https://datachart.500.com/kl8/zoushi/newinc/jbzs_redblue.php?from=2025208&to=2025237"
    
    try:
        # 发送请求
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }
        response = requests.get(url, headers=headers, timeout=30)
        response.raise_for_status()
        
        # 解析HTML内容
        soup = BeautifulSoup(response.text, 'html.parser')
        
        # 根据实际网页结构调整选择器
        # 这里只是一个示例，需要根据实际网页结构调整
        data_rows = soup.select('.kj_table tr')
        
        records = []
        for row in data_rows[1:]:  # 跳过表头
            cells = row.find_all('td')
            if len(cells) >= 22:  # 期号、日期 + 20个号码
                issue = cells[0].text.strip()      # 期号
                date = cells[1].text.strip()       # 开奖日期
                numbers = [int(cells[i].text.strip()) for i in range(2, 22)]  # 20个号码
                
                # 处理日期格式，确保是标准格式
                if date:
                    # 如果日期包含星期几的中文字符，移除它们
                    import re
                    pattern = r'(\d{4}-\d{2}-\d{2})\([一二三四五六日]\)'
                    match = re.match(pattern, date)
                    if match:
                        date = match.group(1)
                
                records.append([issue, date] + numbers)
        
        # 保存到CSV文件
        if records:
            # 读取现有数据（如果文件存在）
            existing_records = []
            if os.path.exists(output_file):
                with open(output_file, 'r', newline='', encoding='utf-8') as csvfile:
                    reader = csv.reader(csvfile)
                    existing_records = list(reader)
                # 移除表头
                if existing_records:
                    existing_records = existing_records[1:]
            
            # 合并现有数据和新数据
            all_records = existing_records + records
            
            # 按期号排序，确保数据是正序的（从旧到新）
            all_records.sort(key=lambda x: int(x[0]))  # 按照期号列（索引为0）进行排序
            
            # 去重，保留最新的数据
            unique_records = []
            seen_issues = set()
            for record in reversed(all_records):  # 从新到旧遍历
                issue = record[0]
                if issue not in seen_issues:
                    seen_issues.add(issue)
                    unique_records.append(record)
            unique_records.reverse()  # 恢复为从旧到新的顺序
            
            # 写入数据
            with open(output_file, 'w', newline='', encoding='utf-8') as csvfile:
                writer = csv.writer(csvfile)
                # 写入表头
                header = ['期号', '开奖日期'] + [f'{i}号' for i in range(1, 21)]
                writer.writerow(header)
                # 写入数据
                writer.writerows(unique_records)
            
            logger.info(f"成功从备用源获取 {len(records)} 期开奖数据并保存到 {output_file}（已按期号正序排列）")
            return len(records)
        else:
            logger.warning("未从备用源获取到有效的开奖数据")
            return 0
            
    except requests.RequestException as e:
        logger.error(f"网络请求失败: {e}")
        return 0
    except Exception as e:
        logger.error(f"数据获取过程中发生错误: {e}")
        return 0

def update_kl8_history_data():
    """更新快乐8历史数据"""
    # 输出文件路径
    output_file = os.path.join(os.path.dirname(__file__), 'kl8_history.csv')
    
    logger.info("开始更新快乐8历史数据...")
    
    # 首先尝试从主数据源获取
    logger.info("尝试从主数据源获取数据...")
    fetched_count = fetch_latest_kl8_data(output_file)
    
    # 如果主数据源没有获取到新数据，可以尝试备用数据源
    if fetched_count == 0:
        logger.info("主数据源未获取到新数据，尝试备用数据源...")
        # fetched_count = fetch_kl8_data_from_alternative_source(output_file)
    
    logger.info("快乐8历史数据更新完成")
    return fetched_count

def create_sample_data():
    """创建示例数据文件（用于测试）"""
    output_file = os.path.join(os.path.dirname(__file__), 'kl8_history.csv')
    
    # 示例数据
    sample_data = [
        ['2020001', '2020/10/28', 6, 8, 15, 27, 28, 30, 37, 38, 39, 44, 46, 49, 51, 52, 59, 70, 75, 77, 78, 80],
        ['2020002', '2020/10/29', 2, 5, 10, 13, 19, 23, 26, 29, 32, 35, 42, 53, 55, 57, 58, 60, 62, 67, 78, 79],
        ['2020003', '2020/10/30', 4, 8, 14, 18, 21, 40, 41, 42, 46, 50, 52, 55, 56, 57, 61, 70, 72, 75, 77, 80]
    ]
    
    # 写入CSV文件
    with open(output_file, 'w', newline='', encoding='utf-8') as csvfile:
        writer = csv.writer(csvfile)
        # 写入表头
        header = ['期号', '开奖日期'] + [f'{i}号' for i in range(1, 21)]
        writer.writerow(header)
        # 写入示例数据
        writer.writerows(sample_data)
    
    logger.info(f"示例数据已保存到 {output_file}")

if __name__ == "__main__":
    # 如果数据文件不存在，创建示例数据
    data_file = os.path.join(os.path.dirname(__file__), 'kl8_history.csv')
    if not os.path.exists(data_file):
        logger.info("数据文件不存在，创建示例数据...")
        create_sample_data()
    
    # 更新数据
    fetched_count = update_kl8_history_data()
    if fetched_count > 0:
        logger.info(f"成功更新 {fetched_count} 条记录")
    else:
        logger.info("未获取到新数据")